﻿// IOCP_Server.cpp : 此文件包含 "main" 函数。程序执行将在此处开始并结束。
//

#include <iostream>
#include <WinSock2.h>
#include <windows.h>
#include <vector>
#include <strsafe.h>
#include <tchar.h>
#include "IOCP_Server.h"
#include "ServerLogAdapter.h"

#pragma comment(lib,"ws2_32.lib")
#pragma comment(lib,"kernel32.lib")

using namespace std;

#define IOCP_DEBUG_PRINT

// 监听端口
const int IOCP_SERVER_PORT = 6000;

HANDLE g_pIOCPHandle = 0;
char g_buffer[1024] = {0};

enum class IO_OPERATION
{
    IO_READ,
    IO_WRITE
};

struct IO_DATA
{
    //可以使用CONTAINING_RECORD，将OVERLAPPED转IO_DATA
    IO_OPERATION opCode;
    OVERLAPPED Overlapped;
    SOCKET client;
    WSABUF wsabuf;
    int nBytes;
};

//////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////

/*

 //************************************************************************
// 函数名   : WSASocketA
// 功能描述 :
// 输入参数 : int af        地址族规范(AF_INET or AF_INET6)
// 输入参数 : int type      套接字的类型规范 AF_INET(SOCK_STREAM -> TCP, SOCK_DGRAM ->UDP)
// 输入参数 : int protocol  使用的协议
                                (af:AF_INET type:SOCK_STREAM protocol:IPPROTO_TCP)
                                (af:AF_INET type:SOCK_DGRAM protocol:IPPROTO_UDP)
// 输入参数 : LPWSAPROTOCOL_INFOA lpProtocolInfo(WSA_FLAG_OVERLAPPED：支持重叠IO)

// 输入参数 : GROUP g
// 输入参数 : DWORD dwFlags 用于指定附加套接字属性的一组标记
// 输出参数 :               重叠套接字可以使用 WSASend, WSASendTo, WSARecv, WSARecvFrom和wsaoctl来进行重叠的I/O操作，这允许多个操作同时启动和进行
// 返回值   : SOCKET WSAAPI
//************************************************************************
SOCKET WSAAPI WSASocketA(
    int                 af,
    int                 type,
    int                 protocol,
    LPWSAPROTOCOL_INFOA lpProtocolInfo,
    GROUP               g,
    DWORD               dwFlags
);




//************************************************************************
// 函数名   : CreateIoCompletionPort
// 功能描述 :
// 输入参数 : HANDLE FileHandle                 打开的文件句柄(必须指向支持重叠I/O的对象)或INVALID_HANDLE_VALUE(创建一个I/O完成端口，而不将其与文件句柄关联)
// 输入参数 : HANDLE ExistingCompletionPort     一个已经存在的I/O完成端口的句柄，或者为空
// 输入参数 : ULONG_PTR CompletionKey           每句柄用户定义的完成键，包含在指定文件句柄的每个I/O完成包中
// 输入参数 : DWORD NumberOfConcurrentThreads   操作系统允许并发处理I/O完成端口的I/O完成包的最大线程数。如果ExistingCompletionPort参数不为空，则忽略此参数。
// 输出参数 :
// 返回值   : HANDLE
//************************************************************************
HANDLE CreateIoCompletionPort(
    HANDLE    FileHandle,
    HANDLE    ExistingCompletionPort,
    ULONG_PTR CompletionKey,
    DWORD     NumberOfConcurrentThreads
);

*/

//************************************************************************
// 函数名   : CreateThread
// 功能描述 : 创建要在调用进程的虚拟地址空间中执行的线程
// 输入参数 : LPSECURITY_ATTRIBUTES lpThreadAttributes  该结构决定返回的句柄是否可以被子进程继承。如果lpThreadAttributes为空，则不能继承句柄。
// 输入参数 : SIZE_T dwStackSize    堆栈的初始大小，以字节为单位。系统将此值舍入到最近的页面。如果该参数为0，则新线程使用可执行文件的默认大小(1MB,64kb倍数)
// 输入参数 : LPTHREAD_START_ROUTINE lpStartAddress     一个指向应用程序定义的函数的指针，该函数将由线程执行。这个指针表示线程的起始地址
// 输入参数 : __drv_aliasesMem LPVOID lpParameter       指向要传递给线程的变量的指针。
// 输入参数 : DWORD dwCreationFlags   控制线程创建的标志。
//                                    0：线程创建后立即执行
//                                    CREATE_SUSPENDED：线程创建时处于挂起状态，在调用ResumeThread函数之前不会运行。
//                                    STACK_SIZE_PARAM_IS_A_RESERVATION： dwStackSize参数指定堆栈的初始保留大小。如果未指定此标志，则dwStackSize指定提交大小。
// 输入参数 : LPDWORD lpThreadId      一个指向变量的指针，用来接收线程标识符。如果该参数为空，则不返回线程标识符。
// 输出参数 :
// 返回值   : HANDLE 
//************************************************************************
//HANDLE CreateThread(
//    LPSECURITY_ATTRIBUTES   lpThreadAttributes,
//    SIZE_T                  dwStackSize,
//    LPTHREAD_START_ROUTINE  lpStartAddress,
//    __drv_aliasesMem LPVOID lpParameter,
//    DWORD                   dwCreationFlags,
//    LPDWORD                 lpThreadId
//);

//////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////

void ErrorHandler(LPTSTR lpszFunction)
{
    // Retrieve the system error message for the last-error code.

    LPVOID lpMsgBuf;
    LPVOID lpDisplayBuf;
    DWORD dw = GetLastError();

    FormatMessage(
        FORMAT_MESSAGE_ALLOCATE_BUFFER |
        FORMAT_MESSAGE_FROM_SYSTEM |
        FORMAT_MESSAGE_IGNORE_INSERTS,
        NULL,
        dw,
        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
        (LPTSTR)&lpMsgBuf,
        0, NULL);

    // Display the error message.

    lpDisplayBuf = (LPVOID)LocalAlloc(LMEM_ZEROINIT,
        (lstrlenW((LPCWSTR)lpMsgBuf) + lstrlen((LPWSTR)lpszFunction) + 40) * sizeof(TCHAR));
    StringCchPrintfW((STRSAFE_LPWSTR)lpDisplayBuf,
        LocalSize(lpDisplayBuf) / sizeof(TCHAR),
        TEXT("%s failed with error %d: %s"),
        lpszFunction, dw, lpMsgBuf);
    MessageBox(NULL, (LPCTSTR)lpDisplayBuf, TEXT("Error"), MB_OK);

    // Free error-handling buffer allocations.

    LocalFree(lpMsgBuf);
    LocalFree(lpDisplayBuf);
}

int GetCpuCoreCount()
{
    //typedef struct _SYSTEM_INFO {
    //    union {
    //        DWORD dwOemId;          // Obsolete field...do not use
    //        struct {
    //            WORD wProcessorArchitecture;
    //            WORD wReserved;
    //        } DUMMYSTRUCTNAME;
    //    } DUMMYUNIONNAME;
    //    DWORD dwPageSize;
    //    LPVOID lpMinimumApplicationAddress;
    //    LPVOID lpMaximumApplicationAddress;
    //    DWORD_PTR dwActiveProcessorMask;
    //    DWORD dwNumberOfProcessors;
    //    DWORD dwProcessorType;
    //    DWORD dwAllocationGranularity;
    //    WORD wProcessorLevel;
    //    WORD wProcessorRevision;
    //} SYSTEM_INFO, * LPSYSTEM_INFO;

    SYSTEM_INFO stSysInfo = {0};
    GetSystemInfo(&stSysInfo);
    //cout << "stSysInfo.dwPageSize = " << stSysInfo.dwPageSize << endl;
    TRACESVR("stSysInfo.dwPageSize = %d", stSysInfo.dwPageSize);
    return stSysInfo.dwNumberOfProcessors;
}

//************************************************************************
// 函数名   : ThreadFuncProc
// 功能描述 : 线程执行函数
// 输入参数 : LPVOID pContext
// 输出参数 :
// 返回值   : DWORD WINAPI 
//************************************************************************
DWORD WINAPI ThreadFuncProc(LPVOID pContext)
{
    //cout << "ThreadID = " << hex << GetCurrentThreadId() << endl; 
    DWORD dwCurThreadID = GetCurrentThreadId();

    while (true)
    {
        DWORD dwIoSize = 0;
        void* lpCompletionKey = NULL;
        LPOVERLAPPED lpOverlapped = NULL;
        BOOL bRet = GetQueuedCompletionStatus(g_pIOCPHandle, &dwIoSize, (PULONG_PTR)&lpCompletionKey, (LPOVERLAPPED*)&lpOverlapped, INFINITE);
        IO_DATA* lpIOContext = CONTAINING_RECORD(lpOverlapped, IO_DATA, Overlapped);
        if (dwIoSize == 0)
        {
            cout << "Client disconnect" << endl;
            int nRet = closesocket(lpIOContext->client);
            cout << "closesocket() nRet=" << nRet << endl;
            delete lpIOContext;
            continue;
        }

#ifdef IOCP_DEBUG_PRINT
        //TRACESVR("[Sub Thread GetQueuedCompletionStatus]dwCurThreadID = %u, dwIoSize = %u, Key = %d, lpIOContext->client = %d, (int)lpIOContext->opCode = %d, bRet = %d", dwCurThreadID, dwIoSize, lpCompletionKey, lpIOContext->client, (int)lpIOContext->opCode, bRet);
        //cout << "[Sub Thread GetQueuedCompletionStatus] dwCurThreadID = " << dwCurThreadID << ", dwIoSize=" << dwIoSize << ", Key =" << lpCompletionKey << ", bRet = " << bRet << ", client = " << lpIOContext->client << ", opCode=" << (int)lpIOContext->opCode << endl;
#endif

        // 客户端刚accept带上来的数据！！！
        // [Sub Thread WSASend] dwCurThreadID = 4fe8, nBytes = 14, nRet = 0, wsabuf = Send Test Count = 0
        // 读写标志位交替
        if (lpIOContext->opCode == IO_OPERATION::IO_READ)
        {
            ZeroMemory(&lpIOContext->Overlapped, sizeof(lpIOContext->Overlapped));
            lpIOContext->wsabuf.buf = g_buffer;
            lpIOContext->wsabuf.len = (ULONG)strlen(g_buffer) + 1;
            lpIOContext->opCode = IO_OPERATION::IO_WRITE;
            lpIOContext->nBytes = (ULONG)strlen(g_buffer) + 1;
            DWORD dwFlags = 0;
            DWORD nBytes = (DWORD)strlen(g_buffer) + 1;
            int nRet = WSASend(lpIOContext->client, &lpIOContext->wsabuf, 1, &nBytes, dwFlags, &(lpIOContext->Overlapped), NULL);
            if (nRet == SOCKET_ERROR && WSAGetLastError() != ERROR_IO_PENDING)
            {
                TRACESVR("WASSend Failed! nErr = %d", WSAGetLastError());
                nRet = closesocket(lpIOContext->client);
                cout << "closesocket() nRet=" << nRet << endl;
                delete lpIOContext;
                continue;
            }
#ifdef IOCP_DEBUG_PRINT
            TRACESVR("[Sub Thread WSASend] dwCurThreadID = %u, nBytes = %d, nRet = %d, wsabuf = %s", dwCurThreadID, nBytes, nRet, lpIOContext->wsabuf.buf);
            //cout << "[Sub Thread WSASend] dwCurThreadID = " << dwCurThreadID << ", nBytes = " << nBytes << ", nRet = " << nRet << ", wsabuf = " << lpIOContext->wsabuf.buf << endl;
#endif
            //memset(g_buffer, NULL, sizeof(g_buffer));
        }
        else if (lpIOContext->opCode == IO_OPERATION::IO_WRITE)
        {
            DWORD dwFlags = 0;
            DWORD nBytes = sizeof(g_buffer);
            lpIOContext->opCode = IO_OPERATION::IO_READ;
            lpIOContext->wsabuf.buf = g_buffer;
            lpIOContext->wsabuf.len = nBytes;
            lpIOContext->nBytes = nBytes;
            ZeroMemory(&lpIOContext->Overlapped, sizeof(lpIOContext->Overlapped));
            int nRet = WSARecv(lpIOContext->client, &lpIOContext->wsabuf, 1, &nBytes, &dwFlags, &(lpIOContext->Overlapped), NULL);
            if (nRet == SOCKET_ERROR && WSAGetLastError() != ERROR_IO_PENDING)
            {
                TRACESVR("WASRecv Failed! nErr = %d", WSAGetLastError());
                nRet = closesocket(lpIOContext->client);
                delete lpIOContext;
                continue;
            }
#ifdef IOCP_DEBUG_PRINT
            TRACESVR("[Sub Thread WSARecv] dwCurThreadID = %u, nBytes = %d, nRet = %d, wsabuf = %s", dwCurThreadID, nBytes, nRet, lpIOContext->wsabuf.buf);
            //cout << "[Sub Thread WSARecv] dwCurThreadID = " << dwCurThreadID << ", nBytes = " << nBytes << ", wsabuf = " << lpIOContext->wsabuf.buf << ", nRet = " << nRet << endl;
#endif
        }
    }

    return 0;
}

int InitializeNetWorlk_Accept()
{
    int nThreadCount = GetCpuCoreCount() * 2;
    //cout << "nThreadCount = " << nThreadCount << endl;

    WSADATA stWsaData = {0};
    int nRet = WSAStartup(MAKEWORD(2, 2), &stWsaData);
    if (nRet != NO_ERROR)
    {
        wprintf(L"WSAStartup failed with error: %ld\n", nRet);
        return 1;
    }

    SOCKET uiSocket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
    TRACESVR("nThreadCount = %d, uiSocket = %u", nThreadCount, uiSocket);
    //cout << "WSASocket() uiSocket=" << uiSocket << endl;
    if (uiSocket == INVALID_SOCKET)
    {
        nRet = WSAGetLastError();
        cout << "Err WSASocket Failed! uiSocket = " << uiSocket << ", nRet = " << nRet << endl;
        return nRet;
    }

    sockaddr_in stSockAddr;
    stSockAddr.sin_family = AF_INET;
    stSockAddr.sin_port = htons(IOCP_SERVER_PORT);
    stSockAddr.sin_addr.S_un.S_addr = htonl(INADDR_ANY);
    nRet = bind(uiSocket, (sockaddr*)&stSockAddr, sizeof(stSockAddr));
    //cout << "bind() nRet=" << nRet << endl;
    if (SOCKET_ERROR == nRet)
    {
        nRet = WSAGetLastError();
        cout << "Err bind Failed! nRet = " << nRet << endl;
        return nRet;
    }

    nRet = listen(uiSocket, SOMAXCONN);
    //cout << "listen() nRet=" << nRet << endl;
    if (nRet == SOCKET_ERROR)
    {
        nRet = WSAGetLastError();
        cout << "Err listen Failed! nRet = " << endl;
        return nRet;
    }

    // 创建IOCP
    g_pIOCPHandle = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, nThreadCount);
    //cout << "CreateIOCP() hIOCP=" << hex << g_pIOCPHandle << endl;
    TRACESVR("g_pIOCPHandle = %p", &g_pIOCPHandle);
    if (NULL == g_pIOCPHandle)
    {
        nRet = GetLastError();
        cout << "Err CreateIoCompletionPort Failed! nRet = " << nRet << endl;
        return nRet;
    }

    //////////////////////////////////////////////////////////////////////////
    // 创建工作线程
    vector<HANDLE> stThreadHandleVect;
    vector<DWORD> stThreadVect;
    stThreadHandleVect.clear();
    stThreadVect.clear();

    HANDLE* pTheradHandler = new HANDLE[nThreadCount];
    DWORD* pTheradList = new DWORD[nThreadCount];

    for (int i = 0; i < nThreadCount; ++i)
    {
        pTheradHandler[i] = NULL;
        pTheradList[i] = 0;

        pTheradHandler[i] = CreateThread(NULL, 0, ThreadFuncProc, 0, 0, &pTheradList[i]);
        //cout << "CreateThread() hThread = " << hex << pTheradHandler[i] << ", dwThreadId = " << pTheradList[i] << endl;
        TRACESVR("CreateThread() hThread = %5d [ %p ]", pTheradList[i], &pTheradHandler[i]);
        if (NULL == pTheradHandler[i])
        {
            nRet = GetLastError();
            cout << "Err CreateThread Failed! i = " << i + 1 << ", nRet = " << nRet << endl;
            continue;
        }
    }

    while (uiSocket)
    {
        // 监听客户端连接
        SOCKET hClient = accept(uiSocket, NULL, NULL);
        cout << "accept() hClient=" << hex << hClient << endl;

        // 绑定IOCP
        HANDLE hIocpTemp = CreateIoCompletionPort((HANDLE)hClient, g_pIOCPHandle, hClient, 0);
        //cout << "CreateIOCP() hIocpTemp=" << hex << hIocpTemp << endl;
        TRACESVR("BIND IOCP() hIocpTemp = %p", &hIocpTemp);
        if (hIocpTemp == NULL)
        {
            int nErr = WSAGetLastError();
            cout << "Bind Socket2IOCP Failed! nErr=" << nErr << endl;
            nRet = closesocket(hClient);
            cout << "closesocket() nRet=" << nRet << endl;
            break; //不退出，会一直失败
        }

        //[Main Thread WSARecv] nBytes = 20, dwFlags = 0, nRet = 0, buf = Send Test Count = 0
        //[Sub Thread WSARecv] dwCurThreadID = 15296, nBytes = 1024, nRet = -1, wsabuf = Send Test Count = 0
        // 主线程收到客户端第一次发送的数据！！！
        // 下面这部分代码相当于把客户端第一次带上来的数据压入完全端口队列中(IOCP)
        // 工作线程弹出消息包(GetQueuedCompletionStatus)会收到这条消息
        // ==>>> 主线程只管accept客户端连接，后面的交互都由工作线程完成！！！！
        //post a recv request
        IO_DATA* pMsgData = new IO_DATA;
        //cout << "data = " << data << endl;
        memset(pMsgData, 0, sizeof(IO_DATA));
        //data->nBytes = 0;
        pMsgData->opCode = IO_OPERATION::IO_READ;
        memset(g_buffer, NULL, sizeof(g_buffer));
        pMsgData->wsabuf.buf = g_buffer;
        pMsgData->wsabuf.len = sizeof(g_buffer);
        pMsgData->client = hClient;
        DWORD nBytes = sizeof(g_buffer), dwFlags = 0;
        int nRet = WSARecv(hClient, &pMsgData->wsabuf, 1, &nBytes, &dwFlags, &(pMsgData->Overlapped), NULL);
        if (nRet == SOCKET_ERROR)
        {
            nRet = WSAGetLastError();
            // 错误代码WSA_IO_PENDING表明重叠操作已经成功启动，稍后将提示完成
            if (ERROR_IO_PENDING != nRet)
            {
                cout << "WASRecv Failed! nRet = " << nRet << endl;
                nRet = closesocket(hClient);
                cout << "closesocket() nRet=" << nRet << endl;
                delete pMsgData;
                break;
            }
        }
        //memset(g_buffer, NULL, sizeof(g_buffer));

#ifdef IOCP_DEBUG_PRINT
    TRACESVR("[Main Thread WSARecv] nBytes = %d, dwFlags = %d, nRet = %d, buf = %s", nBytes, dwFlags, nRet, pMsgData->wsabuf.buf);
#endif
    }

    // 线程回收...
    for (int i = 0; i < nThreadCount; i++)
    {
        if (pTheradHandler[i] != NULL)
        {
            CloseHandle(pTheradHandler[i]);
        }
    }

    // 释放线程变量
    delete[] pTheradHandler;
    delete[] pTheradList;

    nRet = closesocket(uiSocket);
    cout << "closesocket() nRet=" << nRet << endl;

    nRet = WSACleanup();
    cout << "WSACleanup() nRet=" << nRet << endl;

    return 0;
}

